# -*- coding: utf-8 -*-
from datetime import timedelta
from jms_dm_bi_day.depend.dim_depend import jms_dim__dim_lmdm_sys_network
from jms_dm_bi_day.depend.ods_depend import jms_ods__yl_tmsnew_tms_shipment_stop, jms_ods__yl_tmsnew_branch_shipment_stop, jms_ods__tab_barscan_loading, jms_ods__tab_barscan_unloading, \
    jms_ods__tab_barscan_send, jms_ods__tab_barscan_bagging, jms_ods__tab_barscan_other, jms_ods__yl_lmdm_sys_network_expand, jms_ods__tab_barscan_arrival, jms_ods__tab_reback_transfer_express, \
    jms_ods__tab_barscan_taking
from jms_dm_bi_day.depend.ods_depend import jms_ods__yl_oms_oms_waybill
from utils.operators.spark_sql_operator import SparkSqlOperator
from jms_dm_bi_day.depend.dwd_depend import jms_dwd__dwd_yl_oms_oms_waybill_incre_dt

jms_dm__dm_cn_breakoff_detail_total_dt = SparkSqlOperator(
    task_id='jms_dm__dm_cn_breakoff_detail_total_dt',
    email=['zhangqinglin@jtexpress.com','yl_bigdata@yl-scm.com'],
    name='jms_dm__dm_cn_breakoff_detail_total_dt_{{ execution_date | date_add(-1) | cst_ds }}',
    sql='jms_dm_bi_day/dm_cn_breakoff_detail_total_dt/execute.hql',
    pool_slots=32,
    executor_cores=8,
    driver_cores=6,
    executor_memory='10G',
    driver_memory='8G',
    num_executors=70,
    # num_executors=10,  # spark.dynamicAllocation.enabled 为 True 时，num_executors 表示最少 Executor 数
    conf={'spark.dynamicAllocation.maxExecutors': 170,
          'spark.sql.sources.partitionOverwriteMode': 'dynamic',
          'spark.dynamicallocation.enabled': 'true',
          'spark.dynamicAllocation.cachedExecutorIdleTimeout': 120,
          'spark.shuffle.service.enabled': 'true',
          'spark.sql.shuffle.partitions': 2000
          },
    hiveconf={'hive.exec.dynamic.partition': 'true',
              'hive.exec.dynamic.partition.mode': 'nonstrict',
              'hive.exec.max.dynamic.partitions.pernode': 5000,
              'hive.exec.max.dynamic.partitions': 30000
              },
    yarn_queue='pro',
    execution_timeout=timedelta(minutes=45),
)

jms_dm__dm_cn_breakoff_detail_total_dt << [jms_ods__yl_oms_oms_waybill,
                                           jms_dwd__dwd_yl_oms_oms_waybill_incre_dt,
                                           jms_ods__yl_tmsnew_tms_shipment_stop,
                                           jms_ods__yl_tmsnew_branch_shipment_stop,
                                           jms_ods__tab_barscan_loading,
                                           jms_ods__tab_barscan_unloading,
                                           jms_ods__tab_barscan_loading,
                                           jms_ods__tab_barscan_arrival,
                                           jms_ods__tab_barscan_send,
                                           jms_ods__tab_barscan_bagging,
                                           jms_ods__tab_barscan_other,
                                           jms_ods__yl_lmdm_sys_network_expand,
                                           jms_dim__dim_lmdm_sys_network,
                                           jms_ods__tab_reback_transfer_express,
                                           jms_ods__tab_barscan_taking, ]
